// coding: utf-8
// @Author : lryself
// @Date : 2021/2/27 22:41
// @Software: GoLand

package main

import (
	"bufio"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"sync"
	"time"
)

var wg sync.WaitGroup

func main() {
	start := time.Now()
	for page:=1;page<=50;page++ {
		//log.Printf("开始爬取第%d页\n", page)
		wg.Add(1)
		go downloadImagePage(fmt.Sprintf("https://www.umei.cc/bizhitupian/diannaobizhi/%d.htm", page))
	}
	wg.Wait()
	log.Println(time.Since(start).Seconds())
}

func downloadImagePage(pageUrl string) {
	imgUrls := parsePage(pageUrl)
	for _, imgUrl := range imgUrls {
		wg.Add(1)
		go downloadImage(imgUrl)
	}
	wg.Done()
}

func parsePage(pageUrl string) []string{
	response,err := http.Get(pageUrl)
	if err != nil {
		log.Println(err)
		return nil
	}
	defer response.Body.Close()
	if response.StatusCode != 200 {
		log.Printf("status code error: %d %s", response.StatusCode, response.Status)
		return nil
	}

	// 创建解析器
	doc, err := goquery.NewDocumentFromReader(response.Body)
	if err != nil {
		log.Println(err)
		return nil
	}
	var imageUrls []string
	doc.Find("body > div.wrap > div.TypeList").Find("img").Each(func(i int, selection *goquery.Selection) {
		imageUrl,_ := selection.Attr("src")
		imageUrls = append(imageUrls, imageUrl)
	})

	return imageUrls
}

func downloadImage(imgUrl string) {
	defer wg.Done()

	imgPath := "F:\\programme\\go\\go_learning\\src\\others\\spider\\download\\"
	imageName := path.Base(imgUrl)
	//log.Println("开始下载",imageName)
	response,err := http.Get(imgUrl)
	if err != nil{
		//log.Println(err)
		return
	}
	defer response.Body.Close()

	//log.Println(imageName, "下载完成！开始写入")
	f, err := os.Create(imgPath + imageName)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	reader := bufio.NewReaderSize(response.Body, 64 * 1024)

	// 获得文件的writer对象
	writer := bufio.NewWriterSize(f, 64 * 1024)

	//start := time.Now()
	_, err = io.CopyBuffer(writer, reader, make([]byte, 64 * 1024))

	//written, err := reader.WriteTo(writer)
	//if err != nil{
	//	log.Println(imageName, err)
	//}

	//bytes := make([]byte, 10 * 1024 * 1024)
	//written := 0
	//for {
	//	ln, err := reader.Read(bytes)
	//
	//	if ln < 0 || err != nil{
	//		break
	//	}
	//	// 注意这里byte数组后的[0:ln]，不然可能会导致写入多余的数据
	//	nw, err := writer.Write(bytes[0:ln])
	//	written += nw
	//	if err != nil{
	//		log.Println(imageName, err)
	//	}
	//	log.Println(written*100/int(response.ContentLength), "%")
	//}

	//log.Println(time.Since(start).Seconds())

	//log.Printf("%s图片下载完成，大小： %dKB\n", imageName, written/1024)
}
